package com.smartdo.crawler.processor;

import cn.hutool.core.util.ReUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.smartdo.crawler.utils.MongodbUtils;
import com.smartdo.crawler.utils.RedisFactory;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * 1.读取redis资源池的商品链接
 * 2.采集并解析商品信息
 * 3.数据保存到mongodb
 *
 * @author zhangcheng
 * @date 2019/4/8 20:43
 */
public class SumaitongCrawler implements PageProcessor {
    public static void main(String[] args) {
        Spider.create(new SumaitongCrawler())
                .thread(10)
                .setScheduler(new SumaitongScheduler())
//                .addUrl("https://www.aliexpress.com/item/Women-Chiffon-Blouse-Top-Fashion-Short-Sleeve-Casual-Shirts-Loose-Tops-Polka-Dot-Print-Thin-Chiffon/32814087190.html?spm=2114.search0103.3.9.44455504Av1JFR&ws_ab_test=searchweb0_0,searchweb201602_7_10065_10068_10890_319_10546_10059_10884_317_10548_10887_10696_321_322_10084_453_10083_454_10103_10618_10307_537_536,searchweb201603_35,ppcSwitch_0&algo_expid=b37129d8-6495-4acf-99d4-20644fcc862b-1&algo_pvid=b37129d8-6495-4acf-99d4-20644fcc862b")
                .runAsync();
    }

    @Override
    public void process(Page page) {

        Map<String, Object> saveMap = new HashMap<>();
        Html html = page.getHtml();
        //标题
        String title = html.xpath("//*[@id=\"j-product-detail-bd\"]/div[1]/div/h1/text()").get();
        saveMap.put("title", title);
        //类目
        List<String> typeList = html.xpath("//div[@class=\"ui-breadcrumb\"]/div//a/text()").all();
        if (typeList.size() > 2) {
            typeList = typeList.subList(2, typeList.size());
        } else {
            typeList = new ArrayList<>();
        }
        saveMap.put("typeList", typeList);
        //图片列表
        List<String> imgList = html.xpath("//*[@id=\"j-image-thumb-list\"]/li//img/@src").all();
        saveMap.put("imgList", imgList);
        //商品ID
        String url = page.getRequest().getUrl();
        String productId = ReUtil.get("\\d+.html", url, 0).replace(".html", "");
        saveMap.put("productId", productId);
        //属性列表 & 品牌名称
        List<Selectable> shuxingList = html.xpath("//*[@id=\"j-product-desc\"]/div[2]/div[2]/ul/li").nodes();
        Map<String, Object> fieldMap = new HashMap<>();
        String brandName = "";
        for (Selectable li : shuxingList) {
            String key = li.xpath("/li/span[1]/text()").get().replace(":", "");
            String value = li.xpath("/li/span[2]/text()").get();
            if ("Brand Name".equals(key)) {
                brandName = value;
            }
            fieldMap.put(key, li.xpath("/li/span[2]/text()").get());
        }
        saveMap.put("fieldMap", fieldMap);
        saveMap.put("brandName", brandName);
        //品牌URL
        String brandUrl = html.xpath("//*[@id=\"j-store-header\"]/div/div/span[1]/a/@href").get();
        saveMap.put("brandUrl", brandUrl);
        //销量
        String orderNum = html.xpath("//*[@id=\"j-order-num\"]/text()").get();
        if (StringUtils.isNotBlank(orderNum)) {
            orderNum = orderNum.replace("orders", "").trim();
        }
        saveMap.put("orderNum", orderNum);
        //品牌ID
        String brandId = ReUtil.get("\\d+", brandUrl, 0).replace("?", "");
        saveMap.put("brandId", brandId);
        //sku信息
        List<String> skuInfoItem = html.xpath("//*[@id=\"j-product-info-sku\"]/dl/dt/text()").all();
        for (int i = 0; i < skuInfoItem.size(); i++) {
            skuInfoItem.set(i, skuInfoItem.get(i).replace(":", "").trim());
        }
        String skuInfoJson = ReUtil.get("skuProducts=(.*?)];", page.getRawText(), 0).replace("skuProducts=", "");
        JSONArray skuInfoArray = null;
        try {
            skuInfoArray = JSONUtil.parseArray(skuInfoJson.substring(0, skuInfoJson.length() - 1));
        } catch (Exception ex) {
            System.out.println(1);
        }
        List<Map<String, Object>> skuInfoList = new ArrayList<>();
        for (int i = 0; i < skuInfoArray.size(); i++) {
            JSONObject bean = skuInfoArray.getJSONObject(i);
            Map<String, Object> map = new HashMap<>();
            Object price = bean.getJSONObject("skuVal").get("actSkuCalPrice");
            price = price == null ? bean.getJSONObject("skuVal").get("skuCalPrice") : price;
            map.put("price", price);
            String skuPropIds = bean.getStr("skuPropIds");
            String[] ids = skuPropIds.split(",");
            List<String> pvalueList = new ArrayList<>();
            for (int j = 0; j < ids.length; j++) {
                String other = "";
                String value = html.xpath("//a[@data-sku-id=\"" + ids[j] + "\"]/@title").get();
                if (StringUtils.isBlank(value)) {
                    value = html.xpath("//a[@data-sku-id=\"" + ids[j] + "\"]//span/text()").get();
                } else {
                    other = html.xpath("//a[@data-sku-id=\"" + ids[j] + "\"]//img/@src").get();
                }
                if (StringUtils.isBlank(other)) {
                    pvalueList.add(skuInfoItem.get(j) + "->" + value);
                } else {
                    pvalueList.add(skuInfoItem.get(j) + "->" + value + "->" + other);
                }
            }
            map.put("pvalueDesc", pvalueList);
            skuInfoList.add(map);
        }
        saveMap.put("skuInfoList", skuInfoList);
        //详情描述URL
        String descUrl = ReUtil.get("detailDesc=(.*?);", page.getRawText(), 0);
        descUrl = descUrl.substring(12, descUrl.length() - 2);
        if (!RedisFactory.sismember("test:sumaitong:descUrlSet", descUrl)) {
            RedisFactory.setListValue("test:sumaitong:descUrl", descUrl);
            RedisFactory.setSetValue("test:sumaitong:descUrlSet", descUrl);
        }
        saveMap.put("descUrl", descUrl);
        saveMap.put("url", url);
        Html descHtml = new Html(HttpRequest.get(descUrl).execute().body());
        List<String> descImgList = descHtml.xpath("//img/@src").all();
        saveMap.put("descContent", descHtml.get());
        saveMap.put("descImgList", descImgList);
        //包装信息
        String weight = "";
        String packageSize = "";
        List<Selectable> packagingList = html.xpath("//*[@id=\"j-shipping-payment-pane\"]/div[2]/div[2]/ul/li").nodes();
        for (Selectable li : packagingList) {
            if ("Package Weight:".equals(li.xpath("//span[1]/text()").get())) {
                weight = li.xpath("//span[2]/text()").get();
            } else if ("Package Size:".equals(li.xpath("//span[1]/text()").get())) {
                packageSize = li.xpath("//span[2]/text()").get();
            }
        }
        saveMap.put("weight", weight);
        saveMap.put("packageSize", packageSize);
        MongodbUtils.getMongoDBInstance().insertOne("test", "sumaitong", saveMap);
        RedisFactory.setSetValue("test:sumaitong:sussces", url);
    }

    @Override
    public Site getSite() {
        return Site.me();
    }
}
